{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector = np.arange(-5,5,0.1)\n",
    "def relu(x):\n",
    "    return max(0.,x)\n",
    "relu = np.vectorize(relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHhZJREFUeJzt3XeYVOXZx/HvDVKEpajgStNVUVRQ2opYA2ossaWoUdA3VhJRoybGEk2ib6LxTWyJosZEjYZqjYm9sRpUVHYBFQFFihRpUhdY2HK/f8xZs8LCzjLlzJnz+1zXXiwzZ865nx2459lnzpyfuTsiIhIdTcIuQEREGkeNW0QkYtS4RUQiRo1bRCRi1LhFRCJGjVtEJGLUuCWnmdlQM3slQ/t+wMx+lYH9mpk9YmYrzez9dO+/gWO/aGY/yuYxJftM53HnDzObCxQC1UA58BJwmbuXJ/HYQcBId+9az30lwX1/S2b77WVmRcAcoJm7V6Vrv8G+zwMucvcj0rnfrRzrSGAM0MPd12XwODcB3d39nEwdQ3KTZtz55xR3LwD6AH2B60OuJ472AOZmsmlLvKlx5yl3Xwy8TKKBA2BmLczsdjP7wsyWBEsFO2bi+Gb2hJktNrPVZvaWmfWsc9+OZnaHmc0L7p8Q1PFWsMkqMys3s0PN7DwzmxA87gEzu32z4zxrZj8Lvr/OzD43s7Vm9omZfS+4fX/gAeDQYL+rgtv/bma/q7Ovi81slpmtMLN/mVnnOve5mf3EzD4LlkBGmJnVM+4Lgb/VOdbNdcew2f6616ljhJk9H9T+npntXWfbnmb2alDXEjP7pZmdAPwS+GFwnKnBtiVmdlHwfRMzuzH4OS81s8fMrF1wX1FQw4+Cfw/LzeyGxjzHEh417jxlZl2BE4FZdW7+P2BfEs28O9AF+HWGSngR2AfYFSgDRtW573agP3AYsDNwDVADHBXc397dC9z93c32OZpEozIAM9sJOA4YG9z/OXAk0A64GRhpZp3cfTrwE+DdYL/tNy/WzI4Gfg+cCXQC5tXZb62TgYOB3sF2x2++H3d/aLNj/WarP6FvOjuoeScSz9ktQV1tgNdILHt1JvG8ve7uLwG3AuOC4/SuZ5/nBV+Dgb2AAuDezbY5AugBHAP8OniRkxynxp1//mlma4H5wFLgN5B4wwy4GLjK3Ve4+1oS//HPykQR7v6wu691943ATUBvM2tnZk2AC4Ar3H2hu1e7+zvBdg35D+AkmjPA6SQa5KLgmE+4+yJ3r3H3ccBnwIAkSx4KPOzuZUEt15OYNRfV2eY2d1/l7l8A46nz20waPO3u7wdr+6Pq7PtkYLG73+HuFcHP9L0k9zkUuNPdZwfvc1wPnGVmO9TZ5mZ33+DuU4GpJF6UJMepceef77p7G2AQsB/QIbi9I9AKKDWzVcFywUvB7Q2pApptdlszoLK+jc2sqZndFixbrAHmBnd1CL5akpgdN4on3kkfS2J2CjCEOjN5M/sfM5tSZ3y9+O/4G9KZxCy79ljlwFckfiuptbjO9+tJzGDTZWv77sZ2/KwC3xhT8P0OJN7Abui4ksPUuPOUu78J/J3EsgTAcmAD0NPd2wdf7YI3MhvyBVC02W178s2mUNcQ4DTgWBLLFrWPtaCOCmDveh6XzClOY4DTzWwP4BDgKYDg738FLgN2CZZDPg6Omcy+F5F4U5Fgf62BXYCFSdTUkHUkXjRr971bIx47n/p/VtDIMQG7k3gRXtKI40sOUuPOb3cD3zazPu5eQ6Kx3WVmuwKYWRcz+8Y6rZm13OzLgHHA+WY2wBL2Ba5iyzXgWm2AjSRmrK1ILMkAENTxMHCnmXUOZueHmlkLYBmJte69tjYgd58cbPc34GV3XxXc1ZpEI1sWjON8EjPuWkuArmbWfCu7Hh2MsU9Qy63Ae+4+d2u1NMJUoGew75Yklo6S9Rywm5ldaYk3l9uY2SHBfUuAomD5qT5jgKvMbE8zK+C/a+JpPdVSsk+NO4+5+zLgMaD2QybXknjja2KwhPEaiTemanUhMSuv+7W3u78MXAc8AqwGXgAeBR7cyqEfIzEbXwh8Akzc7P6rgY+AD4AVJN40beLu60m8Kfd2sNwxcCv7H0NiNj+6zlg/Ae4A3iXR0A4E3q7zmDeAacBiM1u++Q7d/XUSP6engC9JzHLTsv7v7p8C/0vi5/0ZMGHbj/jGY9cC3wZOIbGs8RmJNxsBngj+/MrMyup5+MPAP0icrTOHxG86l2/HECTH6AM4IiIRoxm3iEjEqHGLiESMGreISMSocYuIRMwODW/SeB06dPCioqJM7Dqj1q1bR+vWrcMuI6viOGaI57g15txWWlq63N2T+UBcZhp3UVERkyZNysSuM6qkpIRBgwaFXUZWxXHMEM9xa8y5zcy29oG2LWipREQkYpKacVviAv1rSVygv8rdizNZlIiIbF1jlkoGu/sWnzgTEZHs0lKJiEjEJPWRdzObA6wkcRGfv7j7FteoMLNhwDCAwsLC/mPHbu36Q7mrvLycgoJ4XdUyjmOGeI5bY85tgwcPLk12GTrZxt3Z3RcFV5V7Fbjc3d/a2vbFxcWus0qiIY5jhniOW2PObWaWdONOaqmkTsLIUuAZkk8VERGRNGuwcZtZ6yD3rvbi8seRuEC9iIgE3p+zgocmzCEbV1xN5qySQuCZIJ91B2B0EFQqIiLAsrUbuWx0Ga1b7MDZA7rRqnlGPtv4tQb37u6zUYCoiEi9qmucK8ZOZvWGSh69YEDGmzZk6CPvIiJxcfdrn/LO51/xh9MPYv9ObbNyTJ3HLSKynUpmLuWeN2ZxRv+unFncLWvHVeMWEdkOi1Zt4KpxU9hvtzb872m9Gn5AGqlxi4g00qaqGi4dXUZltXPf0H7s2LxpVo+vNW4RkUa67cUZTP5iFSOG9GOvjtn/ZKZm3CIijfDiR1/y8NtzOO+wIk46qFMoNahxi4gkae7ydfziyQ/p3a09v/zO/qHVocYtIpKEispqLhlVRtMmxoghfWm+Q3jtU2vcIiJJuOlf05j+5RoeOe9guu7UKtRaNOMWEWnAU6ULGPvBfC4dvDeD99s17HLUuEVEtmXm4rXc8M+PGLjXzlx17L5hlwOocYuIbFX5xiouGVVKm5bN+PPZfdmhaW60zNyoQkQkx7g71z/9EXOXr+PPZ/Vl1zYtwy7pa2rcIiL1GDlxHv+euoifH9eDQ/feJexyvkGNW0RkM1Pnr+K3z01ncI+OXPKtvcMuZwtq3CIidaxav4nho8ro2KYFd57ZhyZNLOyStqDzuEVEAjU1zs8fn8rStRU88ZPD2Kl187BLqpdm3CIigb+8NZvXZyzlhu/sT59u7cMuZ6vUuEVEgImzv+L2V2Zy0oGd+NFhRWGXs01q3CISe0vXVnD5mMnssXMrbvvBgQTh6DlLa9wiEmvVNc4VY6awtqKSf1w4gDYtm4VdUoPUuEUk1u569VPenf0Vfzz9IPbbLTthv6nSUomIxNb4mUu5d/wszizuyhlZDPtNlRq3iMTSwhDDflOlxi0isbOpqoZLR5VRVe3cf05/WjbLbthvqrTGLSKx8/sXpzNl/iruG9qPPTu0DrucRtOMW0Ri5YWPvuSRt+dy3mFFfOfAcMJ+U6XGLSKxMWf5Oq558kP6hBz2myo1bhGJhYrKai4ZWcoOTY0RQ/uFGvabKq1xi0gs/ObZacxYvJZHzj+YLu13DLuclET3JUdEJElPli5g3KQg7LdH+GG/qUq6cZtZUzObbGbPZbIgEZF0mrF4DTfmWNhvqhoz474CmJ6pQkRE0m1DlTN8VFnOhf2mKqlRmFlX4CTgb5ktR0QkPdydRz7eyNzl67jn7NwK+01Vsi8/dwPXADUZrEVEJG3+MXEe7y+u5hfH78fAvXIr7DdVDZ5VYmYnA0vdvdTMBm1ju2HAMIDCwkJKSkrSVWPWlJeXR7LuVMRxzBDPccdpzLNXVXPLexX02snp4V9QUjI/7JLSytx92xuY/R44F6gCWgJtgafd/ZytPaa4uNgnTZqUzjqzoqSkhEGDBoVdRlbFccwQz3HHZcyr1m/ipD9PAOD6fsbJxw0OuaLkmFmpuxcns22DSyXufr27d3X3IuAs4I1tNW0RkbDU1Dg/e3wqy9Zu5L6h/ShonttJNtsrP95iFREBHnjrc96YsZQbT96f3jkc9puqRn1y0t1LgJKMVCIikoKJs7/i9pdncvJBnTh34B5hl5NRmnGLSOTVhv0WdWjNbT84KOfDflOla5WISKRVVdd8HfY78sJDKGiR/20t/0coInntrtcSYb+3n9GbHru1CbucrNBSiYhE1vgZSxkx/nN+WNyN0/t3DbucrFHjFpFIWrhqA1c9PoX9O7Xl5tN6hl1OVqlxi0jk1A37vW9ov8iF/aZKa9wiEjm3vpAI+70/omG/qdKMW0Qi5fkPv+Tv78zl/MOLODGiYb+pUuMWkciYvayca5/6kL67t+f6E6Mb9psqNW4RiYSKymqGjyqjWVNjxJBoh/2mSmvcIhIJv372Y2YuWcsj5x1M54iH/aYqvi9ZIhIZT0yaz+OTFnDZ4O4MyoOw31SpcYtITpuxeA2/evZjDtt7F67Mk7DfVKlxi0jOWltRySUjy2jbshl/OqsvTZvk98WjkqU1bhHJSe7OdU9/xBcr1jP6okPo2KZF2CXlDM24RSQnPfrOXJ7/8EuuPq4Hh+RZ2G+q1LhFJOdM/mIlt7wwnWP225UfH7VX2OXkHDVuEckpK9dt4rLRk9m1TUvuOLM3TbSuvQWtcYtIzkiE/U5h2dqNPPGTQ2nfqnnYJeUkzbhFJGfc/+bnjJ+5LO/DflOlxi0iOeHdz7/ijldmckrvznkf9psqNW4RCV3dsN/ff//AvA/7TZXWuEUkVFXVNfx0zGTKN1Yy6qJ4hP2mSj8hEQnVXa99ysTZK7gjRmG/qdJSiYiEpjbs96yDu/GDGIX9pkqNW0RCURv2e0Cnttx0arzCflOlxi0iWVcb9lsd07DfVGmNW0Syrm7Yb1EMw35TpRm3iGRVbdjvBYfvGduw31SpcYtI1tSG/fbbvT3Xnbhf2OVElhq3iGTFhk3/Dfu9N+Zhv6nSGreIZIXCftOnwZc8M2tpZu+b2VQzm2ZmN2ejMBHJH49/MJ8nShdwucJ+0yKZGfdG4Gh3LzezZsAEM3vR3SdmuDYRyQPTv0yE/R7efReuUNhvWjTYuN3dgfLgr82CL89kUSKSH9ZWVDJ8VBntdmzG3T9U2G+6WKIvN7CRWVOgFOgOjHD3a+vZZhgwDKCwsLD/2LFj01xq5pWXl1NQUBB2GVkVxzFDPMed7TG7OyOmbKRsaTXXHtySHjtn/0M2UXqeBw8eXOruxUlt7O5JfwHtgfFAr21t179/f4+i8ePHh11C1sVxzO7xHHe2x/zwhNm+x7XP+f0ls7J63Lqi9DwDkzzJXtyo83HcfRVQApzQqJcSEYmVyV+s5NYXpnPs/rsy7EiF/aZbMmeVdDSz9sH3OwLHAjMyXZiIRFNt2G9h25bccUYfhf1mQDJnlXQCHg3WuZsAj7v7c5ktS0SiqKbGuSoI+33ykkNp16pZ2CXlpWTOKvkQ6JuFWkQk4u5/83NKZi7jt6f15KCuCvvNFH3mVETS4p3Pl38d9nuOwn4zSo1bRFK2dE0FPx0zRWG/WaJrlYhISqqqa7hcYb9ZpZ+wiKTkzlc/5b05K7jzTIX9ZouWSkRku70xYwn3lXzO2QO68f1+CvvNFjVuEdkuC1au56pxUzmgU1t+c4rCfrNJjVtEGm1jVTWXjiqjpkZhv2HQGreINNqtz09n6oLVPHCOwn7DoBm3iDTKv6cu4tF353HhEXtyQi+F/YZBjVtEkvb5snKuU9hv6NS4RSQpGzZVM3xkGc13aMK9Q/rRrKnaR1i0xi0iSfnVsx/z6dK1PHr+AIX9hkwvmSLSoMc/mM+TpQu4/Oh9OGrfjmGXE3tq3CKyTZ8sSoT9HtG9A1ccs0/Y5Qhq3CKyDWsqKhk+qpT2rZpx91l9FPabI7TGLSL1cneuffJD5q/cwNhhA+lQ0CLskiSgGbeI1OuRt+fy4seLueb4HhxctHPY5UgdatwisoWyr8N+Cxl2lMJ+c40at4h8w4p1m7h0VBm7tWvJHWf0VihCDtIat4h8rabGuXLcFL4q38RTlxymsN8cpRm3iHxtxPhZvPXpMn51ygEc2LVd2OXIVqhxiwgA78xazl2vfcqpvTtzziG7h12ObIMat4iwZE0FPx07mT0V9hsJWuMWibnasN91G6sZffFAWivsN+fpGRKJudtf+ZT3g7DffQsV9hsFWioRibHXpy/hgTcV9hs1atwiMTV/xXp+9rjCfqNIjVskhjZWVXPp6DJq3Ln/HIX9Ro3WuEVi6Jbnp/PhgtU8cE5/9thFYb9Roxm3SMz8e+oiHnt3HhcdsScn9Not7HJkO6hxi8RIbdhv/z124lqF/UZWg43bzLqZ2Xgzm25m08zsimwUJiLptbHKuWRkKS2aNeXeIX0V9hthyaxxVwE/d/cyM2sDlJrZq+7+SYZrE5E0cXce+2QTny2t4tHzB9CpncJ+o6zBl1x3/9Ldy4Lv1wLTgS6ZLkxE0mfcB/N5e1GVwn7zhLl78hubFQFvAb3cfc1m9w0DhgEUFhb2Hzt2bPqqzJLy8nIKCgrCLiOr4jhmiNe4562p5rcTK9i7rXPtIa1pEqPrkETpeR48eHCpuxcns23SjdvMCoA3gVvc/eltbVtcXOyTJk1Kar+5pKSkhEGDBoVdRlbFccwQn3GvqajklHsmUFFZzQ39m3Lq8YPDLimrovQ8m1nSjTupdyfMrBnwFDCqoaYtIrnB3bnmiQ9ZsHID9w7pR9sW8Zlp57tkziox4CFgurvfmfmSRCQdHn57Li9NW8y1JyjsN98kM+M+HDgXONrMpgRf38lwXSKSgtJ5K/n9C9P59gGFXHykwn7zTYOnA7r7BEC/Y4lExIp1m7hsdBmd2rfkdoX95iVdq0Qkj2wR9rujwn7zkT46JZJHasN+f62w37ymxi2SJ2rDfk/r05mhCvvNa2rcInmgNux3r44F3Po9hf3mO61xi0Rc3bDfMRf3U9hvDOgZFom42rDfu37Ym30U9hsLWioRibD/hv3uzvf6Kuw3LtS4RSJq/or1XDVuCr26tOU3pxwQdjmSRWrcIhFUG/brwH1D+ivsN2a0xi0SQb97LhH2++C5/dl9l1ZhlyNZphm3SMT8a+oi/jFxHhcfuSfH9VTYbxypcYtEyKylibDf4j124poTFPYbV2rcIhGxflMVw0eV0rJZU+5R2G+saY1bJALcnRv/+TGfLS3nsQsU9ht3eskWiYBxH8zn6bKFXHHMPhy5j8J+406NWyTHTVu0ml//axpH7tOBy4/eJ+xyJAeocYvksDUVlQwfVcbOrZpz9w/70LSJLh4lWuMWyVl1w37HDRvILgUtwi5JcoRm3CI56qEJc3hp2mKuO2E/ihX2K3WocYvkoNJ5K7jtxRkcd0AhFx25Z9jlSI5R4xbJMYmw38l0br8jf1TYr9RDa9wiOeTrsN91m3haYb+yFZpxi+SQe4Ow35tO6UmvLgr7lfqpcYvkiLeDsN/v9e3C2QO6hV2O5DA1bpEcsGRNBVeMnUz3jgXc8r1eWteWbdIat0jIKqtruGx0Ges3VTN2WD9aNdd/S9k2/QsRCdntL8/kg7kr+dNZfei+q8J+pWFaKhEJ0aufLOEvb81m6CG7c1qfLmGXIxGhxi0Skvkr1vPzxxNhv786WWG/kjw1bpEQVFRWM3yUwn5l+2iNWyQEv3v+Ez5aqLBf2T4NzrjN7GEzW2pmH2ejIJF89+yUhYyc+AXDjtpLYb+yXZJZKvk7cEKG6xCJhVlL13L90x9xcNFO/OL4HmGXIxHVYON297eAFVmoRSSvrd9UxSUjy9ixWVPuObufwn5lu2mNWyQL3J0bn/mYWcvK+ccFh7Bbu5ZhlyQRZu7e8EZmRcBz7t5rG9sMA4YBFBYW9h87dmyaSsye8vJyCgoKwi4jq+I4Zsj+uEvmV/L3aZv4bvdmfLd786wdt644PtdRGvPgwYNL3b04mW3TNuN29weBBwGKi4t90KBB6dp11pSUlBDFulMRxzFDdsf98cLVjH7tHY7cpwN3nj+AJiHlRsbxuc7XMWuRTSSD1lRUcuno/4b9htW0Jb8kczrgGOBdoIeZLTCzCzNflkj0uTu/eGIqC1duYMTQvgr7lbRpcKnE3c/ORiEi+eahCXN4edoSbjxpf/rvobBfSR8tlYhkQG3Y7/E9C7nwCIX9SnqpcYuk2VflG7l01GS67LQjfzhdYb+SfjqPWySNqoOw3xXrN/HMcIX9SmZoxi2SRve+MYv/fLacm0/tSc/OCvuVzFDjFkmTCZ8t5+7XP+X7fbtw1sEK+5XMUeMWSYPFqxNhv/vsWsDvFPYrGabGLZKiyuoaLh9TxobKau4bqrBfyTz9CxNJkcJ+Jds04xZJQW3Y7zkDFfYr2aPGLbKdasN+D+zSTmG/klVq3CLboTbsF+C+of1osYPCfiV7tMYtsh1qw37/+j/FdNtZYb+SXZpxizRSbdjvj4/ai28fUBh2ORJDatwijVA37Pdqhf1KSNS4RZJUG/bbqnlT7h2isF8Jj9a4RZLg7twQhP2OvPAQCtsq7FfCoymDSBLGvD+fZyYv5Kpj9+Xw7h3CLkdiTo1bpAEfL1zNTf+exlH7duSywd3DLkdEjVtkW1ZvqGT4qDJ2aa2wX8kdWuMW2YrasN9FqzYw7scD2bl187BLEgE04xbZqocmzOGVT5Zw3Yn7KexXcooat0g9asN+T+i5m8J+JeeocYts5hthv2ccpFAEyTla4xapo27Y79OXHEbblgr7ldyjGbdIHfe88dnXYb+9uijsV3KTGrdI4D+fLeNPr3/G9/sp7Fdymxq3CPDl6g1cOXZKIuz3uwr7ldymxi2xV1ldw2WjJwdhv/0V9is5T/9CJfb+8NIMSuet5M9n96X7rgVhlyPSIM24JdZenraYv/5nDucO3INTe3cOuxyRpKhxS2x98dV6rn5iKgd1bceNJ+8fdjkiSVPjlljaVO1cMqoUA0YMUdivREtSjdvMTjCzmWY2y8yuy3RRIpk2esYmpi1aw51n9lHYr0ROg43bzJoCI4ATgQOAs83sgEwXJpIp/5y8kJL5Vfz4W3txrMJ+JYKSOatkADDL3WcDmNlY4DTgk3QXc8o9E6iorE73bpO2bv16Wpe9GdrxwxDHMc9bsZ59d2rCL45T2K9EUzKNuwswv87fFwCHbL6RmQ0DhgEUFhZSUlLS6GIKaipoEeLnHgpa1NDUNoRXQAjiOOaBuzXh+M6VTPjPW2GXklXl5eXb9f8yyvJ1zMk07vpaqW9xg/uDwIMAxcXFPmjQoEYXsx0PSauSkhK2p+4oi+OYIZ7j1pjzRzJvTi4A6l64oSuwKDPliIhIQ5Jp3B8A+5jZnmbWHDgL+FdmyxIRka1pcKnE3avM7DLgZaAp8LC7T8t4ZSIiUq+krlXi7i8AL2S4FhERSYI+OSkiEjFq3CIiEaPGLSISMWrcIiIRY+5bfJYm9Z2aLQPmpX3HmdcBWB52EVkWxzFDPMetMee2Pdy9YzIbZqRxR5WZTXL34rDryKY4jhniOW6NOX9oqUREJGLUuEVEIkaN+5seDLuAEMRxzBDPcWvMeUJr3CIiEaMZt4hIxKhxi4hEjBp3PczsajNzM+sQdi3ZYGZ/NLMZZvahmT1jZu3DrilT4hh8bWbdzGy8mU03s2lmdkXYNWWLmTU1s8lm9lzYtaSTGvdmzKwb8G3gi7BryaJXgV7ufhDwKXB9yPVkRIyDr6uAn7v7/sBA4NKYjBvgCmB62EWkmxr3lu4CrqGeeLZ85e6vuHtV8NeJJFKO8tHXwdfuvgmoDb7Oa+7+pbuXBd+vJdHIuoRbVeaZWVfgJOBvYdeSbmrcdZjZqcBCd58adi0hugB4MewiMqS+4Ou8b2B1mVkR0Bd4L9xKsuJuEpOwmrALSbekghTyiZm9BuxWz103AL8EjstuRdmxrXG7+7PBNjeQ+LV6VDZry6Kkgq/zlZkVAE8BV7r7mrDrySQzOxlY6u6lZjYo7HrSLXaN292Pre92MzsQ2BOYamaQWC4oM7MB7r44iyVmxNbGXcvMfgScDBzj+Xtyf2yDr82sGYmmPcrdnw67niw4HDjVzL4DtATamtlIdz8n5LrSQh/A2QozmwsUu3tUriy23czsBOBO4FvuvizsejLFzHYg8ebrMcBCEkHYQ/I9Q9USM5FHgRXufmXY9WRbMOO+2t1PDruWdNEatwDcC7QBXjWzKWb2QNgFZULwBmxt8PV04PF8b9qBw4FzgaOD53dKMBOViNKMW0QkYjTjFhGJGDVuEZGIUeMWEYkYNW4RkYhR4xYRiRg1bhGRiFHjFhGJmP8HV0Oj9MD5It8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vector = np.arange(-5,5,0.1)\n",
    "plt.plot(vector, relu(vector))\n",
    "plt.grid()\n",
    "plt.title(\"ReLU activation function\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-281cfb583252>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use urllib or similar directly.\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting ./data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting ./data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"./data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28 \n",
    "n_hidden1 = 350\n",
    "n_hidden2 = 200 \n",
    "n_hidden3 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input layer \n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "# target values\n",
    "y = tf.placeholder(tf.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden1 = fully_connected(X, n_hidden1)\n",
    "hidden2 = fully_connected(hidden1, n_hidden2)\n",
    "hidden3 = fully_connected(hidden2, n_hidden3)\n",
    "logits = fully_connected(hidden3, n_outputs, activation_fn=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                   labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct = tf.nn.in_top_k(predictions=logits, targets=y, k=1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= Epoch: 1 ========\n",
      "Train accuracy: 0.825 | Test accuracy: 0.8943\n",
      "--------------------------------------------------\n",
      "======= Epoch: 2 ========\n",
      "Train accuracy: 0.8875 | Test accuracy: 0.9142\n",
      "--------------------------------------------------\n",
      "======= Epoch: 3 ========\n",
      "Train accuracy: 0.9375 | Test accuracy: 0.925\n",
      "--------------------------------------------------\n",
      "======= Epoch: 4 ========\n",
      "Train accuracy: 0.9625 | Test accuracy: 0.9329\n",
      "--------------------------------------------------\n",
      "======= Epoch: 5 ========\n",
      "Train accuracy: 0.9125 | Test accuracy: 0.939\n",
      "--------------------------------------------------\n",
      "======= Epoch: 6 ========\n",
      "Train accuracy: 0.9875 | Test accuracy: 0.9448\n",
      "--------------------------------------------------\n",
      "======= Epoch: 7 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9475\n",
      "--------------------------------------------------\n",
      "======= Epoch: 8 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9512\n",
      "--------------------------------------------------\n",
      "======= Epoch: 9 ========\n",
      "Train accuracy: 0.9375 | Test accuracy: 0.9559\n",
      "--------------------------------------------------\n",
      "======= Epoch: 10 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9532\n",
      "--------------------------------------------------\n",
      "======= Epoch: 11 ========\n",
      "Train accuracy: 0.925 | Test accuracy: 0.9597\n",
      "--------------------------------------------------\n",
      "======= Epoch: 12 ========\n",
      "Train accuracy: 0.9375 | Test accuracy: 0.9608\n",
      "--------------------------------------------------\n",
      "======= Epoch: 13 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9622\n",
      "--------------------------------------------------\n",
      "======= Epoch: 14 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9648\n",
      "--------------------------------------------------\n",
      "======= Epoch: 15 ========\n",
      "Train accuracy: 0.95 | Test accuracy: 0.9653\n",
      "--------------------------------------------------\n",
      "======= Epoch: 16 ========\n",
      "Train accuracy: 0.9875 | Test accuracy: 0.9665\n",
      "--------------------------------------------------\n",
      "======= Epoch: 17 ========\n",
      "Train accuracy: 0.9375 | Test accuracy: 0.9654\n",
      "--------------------------------------------------\n",
      "======= Epoch: 18 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.968\n",
      "--------------------------------------------------\n",
      "======= Epoch: 19 ========\n",
      "Train accuracy: 0.975 | Test accuracy: 0.9697\n",
      "--------------------------------------------------\n",
      "======= Epoch: 20 ========\n",
      "Train accuracy: 1.0 | Test accuracy: 0.9699\n",
      "--------------------------------------------------\n",
      "Done Trainning!\n",
      "\n",
      "=====================\n",
      "\n",
      "Using the network to make individual predictions\n",
      "Actual | Predicted\n",
      "=====================\n",
      "   7   |     7\n",
      "   2   |     2\n",
      "   1   |     1\n",
      "   0   |     0\n",
      "   4   |     4\n",
      "   1   |     1\n",
      "   4   |     4\n",
      "   9   |     9\n",
      "   5   |     6\n",
      "   9   |     9\n",
      "   0   |     0\n",
      "   6   |     6\n",
      "   9   |     9\n",
      "   0   |     0\n",
      "   1   |     1\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    ## Initializing the variables\n",
    "    tf.global_variables_initializer().run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"======= Epoch: {} ========\".format(epoch+1))\n",
    "        print(\"Train accuracy:\", acc_train, \"| Test accuracy:\", acc_test)\n",
    "        print(50*\"-\")\n",
    "    print(\"Done Trainning!\")\n",
    "\n",
    "    ## Producing individual predictions\n",
    "    print(\"\\n=====================\\n\")\n",
    "    print(\"Using the network to make individual predictions\")\n",
    "    n_pred = 15\n",
    "    X_new = mnist.test.images[:n_pred]\n",
    "    Z = logits.eval(feed_dict={X: X_new})\n",
    "    y_pred = np.argmax(Z, axis=1)\n",
    "    print(\"Actual | Predicted\")\n",
    "    print(\"=====================\")\n",
    "    for obs, pred in zip(mnist.test.labels[:n_pred], y_pred):\n",
    "        print(\"{: >4}   |{: >6}\".format(obs, pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
